Merge pull request #58 from rishabhjain/master

Deployment

Andrew Cantino 11 ans auparavant
Parent
Commettre
7ca0629539

+ 1 - 1
app/models/agents/event_formatting_agent.rb

@@ -21,7 +21,7 @@ module Agents
21 21
                  message => "Today's conditions look like <$.conditions> and high temperaure is going to be <$.high.celsius> degree Celsius.""
22 22
                  subject => "$.data"
23 23
 
24
-            JSONPaths must be between < and > . Make sure you dont use these symbols anywhere else. You can add as many keys as you like.
24
+            JSONPaths must be between < and > . Make sure you dont use these symbols anywhere else.
25 25
 
26 26
             Event generated by Event Formatting Agent will be like
27 27
 

+ 43 - 0
app/models/agents/post_agent.rb

@@ -0,0 +1,43 @@
1
+module Agents
2
+    class PostAgent < Agent
3
+        cannot_be_scheduled!
4
+
5
+        description <<-MD
6
+            Post Agent receives events from other agents and send those events as the contents of a post request to a specified url. `post_url` field must specify where you would like to receive post requests and do not forget to include URI scheme(`http` or `https`)
7
+        MD
8
+
9
+        event_description <<-MD
10
+            Does not produce any event.
11
+        MD
12
+
13
+        def default_options
14
+            {
15
+                :post_url => "http://www.example.com",
16
+                :expected_receive_period_in_days => 1
17
+            }
18
+        end
19
+
20
+        def working?
21
+            last_receive_at && last_receive_at > options[:expected_receive_period_in_days].to_i.days.ago
22
+        end
23
+
24
+        def validate_options
25
+            unless options[:post_url].present? && options[:expected_receive_period_in_days].present? 
26
+                errors.add(:base, "post_url and expected_receive_period_in_days are required fields")
27
+            end
28
+        end
29
+
30
+        def post_event(uri,event)
31
+            req = Net::HTTP::Post.new(uri.request_uri)
32
+            req.form_data = event
33
+            Net::HTTP.start(uri.hostname, uri.port, :use_ssl => uri.scheme == "https") { |http| http.request(req) }
34
+        end
35
+
36
+        def receive(incoming_events)
37
+            incoming_events.each do |event|
38
+                uri = URI options[:post_url]
39
+                post_event uri, event.payload
40
+            end
41
+        end
42
+    end
43
+end

+ 0 - 1
app/models/agents/twilio_agent.rb

@@ -78,7 +78,6 @@ module Agents
78 78
     end
79 79
 
80 80
     def receive_webhook(params)
81
-      create_event :payload => params
82 81
       if memory[:pending_calls].has_key? params[:secret].to_sym
83 82
         response = Twilio::TwiML::Response.new {|r| r.Say memory[:pending_calls][params[:secret].to_sym], :voice => 'woman'}
84 83
         memory[:pending_calls].delete params[:secret].to_sym

+ 10 - 0
deployment/Cheffile

@@ -0,0 +1,10 @@
1
+#!/usr/bin/env ruby
2
+#^syntax detection
3
+
4
+site 'http://community.opscode.com/api/v1'
5
+
6
+cookbook 'runit'
7
+cookbook 'git', :git => 'git://github.com/opscode-cookbooks/git.git'
8
+cookbook 'nginx', :git => 'git://github.com/opscode-cookbooks/nginx.git'
9
+cookbook 'mysql', :git => 'git://github.com/opscode-cookbooks/mysql.git'
10
+cookbook 'nodejs', :git => 'git://github.com/mdxp/nodejs-cookbook.git'

+ 39 - 0
deployment/Vagrantfile

@@ -0,0 +1,39 @@
1
+# -*- mode: ruby -*-
2
+# vi: set ft=ruby :
3
+
4
+Vagrant.configure("2") do |config|
5
+  config.omnibus.chef_version = :latest
6
+  config.vm.define :vb do |vb|
7
+    vb.vm.box = "precise32"
8
+    vb.vm.box_url = "http://files.vagrantup.com/precise32.box"
9
+    vb.vm.network :forwarded_port, host: 3000, guest: 3000
10
+
11
+    vb.vm.provision :chef_solo do |chef|
12
+      chef.roles_path = "roles"
13
+      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
14
+      chef.add_role("huginn_development")
15
+    end
16
+  end
17
+
18
+  config.vm.define :ec2 do |ec2|
19
+    ec2.vm.box = "dummy"
20
+    ec2.vm.box_url = "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box"
21
+
22
+    ec2.vm.provider :aws do |aws, override|
23
+      aws.access_key_id = ""
24
+      aws.secret_access_key = ""
25
+      aws.keypair_name = ""
26
+      aws.region = "us-east-1"
27
+      aws.ami = "ami-d0f89fb9"
28
+
29
+      override.ssh.username = "ubuntu"
30
+      override.ssh.private_key_path = ""
31
+    end
32
+    ec2.vm.provision :chef_solo do |chef|
33
+      chef.roles_path = "roles"
34
+      chef.cookbooks_path = ["cookbooks", "site-cookbooks"]
35
+      chef.add_role("huginn_production")
36
+    
37
+    end
38
+  end
39
+end

+ 0 - 0
deployment/nodes/.gitignore


+ 29 - 0
deployment/roles/huginn_development.json

@@ -0,0 +1,29 @@
1
+{
2
+
3
+"name" : "huginn_development",
4
+
5
+"chef_type" : "role",
6
+
7
+"json_class" : "Chef::Role",
8
+
9
+"description" : "Huginn Development Environment",
10
+
11
+"default_attributes" : {
12
+    "mysql" : {
13
+      "server_root_password" : "",
14
+      "server_repl_password" : "",
15
+      "server_debian_password" : ""
16
+    },
17
+    "nginx" : {
18
+       "init_style" : "upstart"
19
+    }
20
+},
21
+
22
+"run_list":[
23
+             "recipe[git]",
24
+             "recipe[apt]",
25
+             "recipe[mysql::server]",
26
+             "recipe[nodejs::install_from_binary]",
27
+             "recipe[huginn_development]"
28
+           ]
29
+}

+ 30 - 0
deployment/roles/huginn_production.json

@@ -0,0 +1,30 @@
1
+{
2
+
3
+"name" : "huginn_production",
4
+
5
+"chef_type" : "role",
6
+
7
+"json_class" : "Chef::Role",
8
+
9
+"description" : "Huginn Production Environment",
10
+
11
+"default_attributes" : {
12
+  "mysql": {
13
+    "server_root_password": "",
14
+    "server_repl_password": "",
15
+    "server_debian_password": ""
16
+  },
17
+    "nginx" : {
18
+       "init_style" : "upstart"
19
+    }
20
+},
21
+
22
+"run_list":[
23
+             "recipe[git]",
24
+             "recipe[apt]",
25
+             "recipe[mysql::server]",
26
+             "recipe[nodejs::install_from_binary]",
27
+             "recipe[nginx]",
28
+             "recipe[huginn_production]"
29
+           ]
30
+}

+ 64 - 0
deployment/site-cookbooks/huginn_development/recipes/default.rb

@@ -0,0 +1,64 @@
1
+include_recipe 'apt'
2
+include_recipe 'build-essential'
3
+
4
+user "huginn" do
5
+  action :create
6
+  system true
7
+  home "/home/huginn"
8
+  password "$6$ZwO6b.6tij$SMa8UIwtESGDxB37NwHsct.gJfXWmmflNbH.oypwJ9y0KkzMkCdw7D14iK7GX9C4CWSEcpGOFUow7p01rQFu5."
9
+  supports :manage_home => true
10
+  gid "sudo"
11
+  shell "/bin/bash"
12
+end
13
+
14
+group "huginn" do
15
+  members ["huginn"]
16
+  action :create
17
+end
18
+
19
+%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl").each do |pkg|
20
+  package pkg do
21
+    action :install
22
+  end
23
+end
24
+
25
+git "/home/huginn/huginn" do
26
+  repository 'git://github.com/cantino/huginn.git'
27
+  reference 'master'
28
+  action :sync
29
+  user "huginn"
30
+end
31
+
32
+gem_package("rake")
33
+gem_package("bundle")
34
+
35
+bash "Setting huginn user with NOPASSWD option" do
36
+  cwd "/etc/sudoers.d"
37
+  code <<-EOH
38
+    touch huginn
39
+    chmod 0440 huginn
40
+    echo "huginn ALL=(ALL) NOPASSWD:ALL" >> huginn
41
+    EOH
42
+end
43
+
44
+bash "huginn dependencies" do
45
+  user "huginn"
46
+  cwd "/home/huginn/huginn"
47
+  code <<-EOH
48
+    export LANG="en_US.UTF-8"
49
+    export LC_ALL="en_US.UTF-8"
50
+    sudo bundle install
51
+    sed s/REPLACE_ME_NOW\!/$(sudo rake secret)/ .env.example > .env
52
+    sudo rake db:create
53
+    sudo rake db:migrate
54
+    sudo rake db:seed
55
+    EOH
56
+end
57
+
58
+bash "huginn has been installed and will start in a minute" do
59
+  user "huginn"
60
+  cwd "/home/huginn/huginn"
61
+  code <<-EOH
62
+    sudo foreman start
63
+    EOH
64
+end

+ 59 - 0
deployment/site-cookbooks/huginn_production/files/default/Gemfile

@@ -0,0 +1,59 @@
1
+source 'https://rubygems.org'
2
+
3
+gem 'rails'
4
+gem 'rake'
5
+gem 'mysql2'
6
+gem 'devise'
7
+gem 'rails_admin'
8
+gem 'kaminari'
9
+gem 'bootstrap-kaminari-views'
10
+gem "rufus-scheduler", :require => false
11
+gem 'json', '>= 1.7.7'
12
+gem 'jsonpath'
13
+gem 'twilio-ruby'
14
+
15
+gem 'delayed_job', :git => 'https://github.com/wok/delayed_job' # Until the YAML issues are fixed in master.
16
+gem 'delayed_job_active_record', "~> 0.3.3" # newer was giving a strange MySQL error
17
+gem "daemons"
18
+# gem "delayed_job_web"
19
+group :production do
20
+  gem 'unicorn'
21
+end
22
+gem 'foreman'
23
+gem 'dotenv-rails', :groups => [:development, :test]
24
+
25
+group :assets do
26
+  gem 'sass-rails',   '~> 3.2.3'
27
+  gem 'coffee-rails', '~> 3.2.1'
28
+  gem 'uglifier', '>= 1.0.3'
29
+  gem 'select2-rails'
30
+  gem 'jquery-rails'
31
+end
32
+
33
+gem 'geokit-rails3'
34
+gem 'kramdown'
35
+gem "typhoeus"
36
+gem 'nokogiri'
37
+gem 'wunderground'
38
+
39
+gem "twitter"
40
+gem 'twitter-stream', '>=0.1.16'
41
+gem 'em-http-request'
42
+
43
+platforms :ruby_18 do
44
+  gem 'system_timer'
45
+  gem 'fastercsv'
46
+end
47
+
48
+group :development do
49
+  gem 'pry'
50
+end
51
+
52
+group :development, :test do
53
+  gem 'rspec-rails'
54
+  gem 'rspec'
55
+  gem 'shoulda-matchers'
56
+  gem 'rr'
57
+  gem 'webmock', :require => false
58
+  gem 'rake'
59
+end

+ 4 - 0
deployment/site-cookbooks/huginn_production/files/default/Procfile

@@ -0,0 +1,4 @@
1
+web: sudo bundle exec unicorn_rails -c config/unicorn.rb
2
+schedule: sudo bundle exec rails runner bin/schedule.rb
3
+twitter: sudo bundle exec rails runner bin/twitter_stream.rb
4
+dj: sudo bundle exec script/delayed_job run

+ 42 - 0
deployment/site-cookbooks/huginn_production/files/default/env.example

@@ -0,0 +1,42 @@
1
+# ==== Required configuration settings for Huginn ====
2
+
3
+# Replace the following with the output from "rake secret"
4
+APP_SECRET_TOKEN=REPLACE_ME_NOW!
5
+
6
+# This is the domain where your Huginn instance will be running.  The default should work
7
+# for development, but it needs to be changed when you deploy to a production environment.
8
+#DOMAIN=localhost:3000
9
+
10
+# Database Setup
11
+DATABASE_ADAPTER=mysql2
12
+DATABASE_ENCODING=utf8
13
+DATABASE_RECONNECT=true
14
+DATABASE_NAME=huginn_production
15
+DATABASE_POOL=5
16
+DATABASE_USERNAME=root
17
+DATABASE_PASSWORD=
18
+#DATABASE_HOST=your-domain-here.com
19
+#DATABASE_PORT=3306
20
+#DATABASE_SOCKET=/tmp/mysql.sock
21
+
22
+# ==== Additional required production settings ====
23
+
24
+# Configure Rails environment.  This should only be needed in production and may cause errors in development.
25
+RAILS_ENV=production
26
+
27
+# Outgoing email settings.  To use Gmail or Google Apps, put your Google Apps domain or gmail.com
28
+# as the SMTP_DOMAIN and your Gmail username and password as the SMTP_USER_NAME and SMTP_PASSWORD.
29
+SMTP_DOMAIN=your-domain-here.com
30
+SMTP_USER_NAME=you@gmail.com
31
+SMTP_PASSWORD=somepassword
32
+SMTP_SERVER=smtp.gmail.com
33
+SMTP_PORT=587
34
+SMTP_AUTHENTICATION=plain
35
+SMTP_ENABLE_STARTTLS_AUTO=true
36
+
37
+# The address from which system emails will appear to be sent.
38
+EMAIL_FROM_ADDRESS=from_address@gmail.com
39
+
40
+# This invitation code will be required for users to signup with your Huginn installation.
41
+# You can see its use in user.rb.
42
+INVITATION_CODE=try-huginn

+ 37 - 0
deployment/site-cookbooks/huginn_production/files/default/nginx.conf

@@ -0,0 +1,37 @@
1
+#worker_process 2;
2
+user huginn huginn;
3
+
4
+events { 
5
+  worker_connections 1024;
6
+  accept_mutex on;
7
+}
8
+
9
+http {
10
+  upstream huginn_server {
11
+    server unix:/home/huginn/shared/tmp/sockets/unicorn.sock;
12
+}
13
+
14
+  server {
15
+    listen 80;
16
+    server_name _;
17
+    keepalive_timeout 5;
18
+    root /home/huginn/current/public;
19
+    try_files $uri/index.html $uri.html $uri @app;
20
+    error_page 500 502 503 504 /500.html;
21
+    location = /500.html {
22
+      root /home/huginn/current/public;
23
+    }
24
+    location @app {
25
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
26
+
27
+      proxy_set_header X-Forwarded-Proto $scheme;
28
+
29
+      proxy_set_header Host $http_host;
30
+
31
+      proxy_redirect off;
32
+
33
+      proxy_pass http://huginn_server;
34
+    }
35
+}
36
+}
37
+

+ 33 - 0
deployment/site-cookbooks/huginn_production/files/default/unicorn.rb

@@ -0,0 +1,33 @@
1
+app_path = "/home/huginn/current"
2
+
3
+worker_processes 2
4
+preload_app true
5
+timeout 180
6
+listen '/home/huginn/shared/tmp/sockets/unicorn.sock'
7
+
8
+working_directory app_path
9
+
10
+rails_env = ENV['RAILS_ENV'] || 'production'
11
+
12
+# Log everything to one file
13
+stderr_path "log/unicorn_out.log"
14
+stdout_path "log/unicorn_err.log"
15
+
16
+# Set master PID location
17
+pid '/home/huginn/shared/tmp/pids/unicorn.pid'
18
+
19
+before_fork do |server, worker|
20
+  ActiveRecord::Base.connection.disconnect!
21
+  old_pid = "#{server.config[:pid]}.oldbin"
22
+  if File.exists?(old_pid) && server.pid != old_pid
23
+    begin
24
+      Process.kill("QUIT", File.read(old_pid).to_i)
25
+    rescue Errno::ENOENT, Errno::ESRCH
26
+      # someone else did our job for us
27
+    end
28
+  end
29
+end
30
+
31
+after_fork do |server, worker|
32
+  ActiveRecord::Base.establish_connection
33
+end

+ 95 - 0
deployment/site-cookbooks/huginn_production/recipes/default.rb

@@ -0,0 +1,95 @@
1
+include_recipe 'apt'
2
+include_recipe 'build-essential'
3
+
4
+user "huginn" do
5
+  system true
6
+  home "/home/huginn"
7
+  password "$6$ZwO6b.6tij$SMa8UIwtESGDxB37NwHsct.gJfXWmmflNbH.oypwJ9y0KkzMkCdw7D14iK7GX9C4CWSEcpGOFUow7p01rQFu5."
8
+  supports :manage_home => true
9
+  shell "/bin/bash"
10
+  gid "sudo"
11
+end
12
+
13
+group "huginn" do
14
+  members ["huginn"]
15
+end
16
+
17
+%w("ruby1.9.1" "ruby1.9.1-dev" "libxslt-dev" "libxml2-dev" "curl" "libshadow-ruby1.8").each do |pkg|
18
+  package("#{pkg}")
19
+end
20
+
21
+gem_package("rake")
22
+gem_package("bundle")
23
+
24
+service "nginx" do
25
+  supports :restart => true, :start => true, :stop => true, :reload => true
26
+  action :nothing
27
+end
28
+
29
+bash "Setting huginn user with NOPASSWD option" do
30
+  cwd "/etc/sudoers.d"
31
+  code <<-EOH
32
+    touch huginn && chmod 0440 huginn 
33
+    echo "huginn ALL=(ALL) NOPASSWD:ALL" >> huginn
34
+  EOH
35
+end
36
+
37
+deploy "/home/huginn" do
38
+  repo "https://github.com/cantino/huginn.git"
39
+  user "huginn"
40
+  group "huginn"
41
+  environment "RAILS_ENV" => "production"
42
+  keep_releases 5
43
+  create_dirs_before_symlink []
44
+  symlinks "log" => "log"
45
+  symlink_before_migrate({})
46
+  rollback_on_error true
47
+  notifies :enable, "service[nginx]"
48
+  notifies :start, "service[nginx]"
49
+  before_symlink do
50
+    %w(config log tmp).each do |dir|
51
+      directory "/home/huginn/shared/#{dir}" do
52
+      owner "huginn"
53
+      group "huginn"
54
+      recursive true
55
+      end
56
+    end
57
+    directory("/home/huginn/shared/tmp/pids")
58
+    directory("/home/huginn/shared/tmp/sockets")
59
+    %w(Procfile unicorn.rb Gemfile nginx.conf).each do |file|
60
+      cookbook_file "/home/huginn/shared/config/#{file}" do
61
+      owner "huginn"
62
+      action :create_if_missing
63
+      end
64
+    end
65
+    cookbook_file "home/huginn/shared/config/.env" do
66
+    source "env.example"
67
+    mode "666"
68
+    owner "huginn"
69
+    action :create_if_missing
70
+    end
71
+  end
72
+  before_restart do
73
+    bash "huginn dependencies" do
74
+      cwd "/home/huginn/current"
75
+      user "huginn"
76
+      group "huginn"
77
+      code <<-EOH
78
+      export LANG="en_US.UTF-8"
79
+      export LC_ALL="en_US.UTF-8"
80
+      ln -nfs /home/huginn/shared/config/Gemfile ./Gemfile
81
+      ln -nfs /home/huginn/shared/config/Procfile ./Procfile
82
+      ln -nfs /home/huginn/shared/config/.env ./.env
83
+      ln -nfs /home/huginn/shared/config/unicorn.rb ./config/unicorn.rb
84
+      sudo cp /home/huginn/shared/config/nginx.conf /etc/nginx/ 
85
+      sudo bundle install
86
+      sed -i s/REPLACE_ME_NOW\!/$(sudo rake secret)/ .env
87
+      sudo rake db:create
88
+      sudo rake db:migrate
89
+      sudo rake db:seed
90
+      sudo foreman export upstart /etc/init -a huginn -u huginn -l log
91
+      sudo start huginn
92
+      EOH
93
+    end
94
+  end
95
+end

+ 6 - 0
deployment/solo.rb

@@ -0,0 +1,6 @@
1
+file_cache_path           "/tmp/chef-solo"
2
+data_bag_path             "/tmp/chef-solo/data_bags"
3
+encrypted_data_bag_secret "/tmp/chef-solo/data_bag_key"
4
+cookbook_path             [ "/tmp/chef-solo/site-cookbooks",
5
+                            "/tmp/chef-solo/cookbooks" ]
6
+role_path                 "/tmp/chef-solo/roles"

+ 71 - 0
spec/models/agents/post_agent_spec.rb

@@ -0,0 +1,71 @@
1
+require 'spec_helper'
2
+
3
+describe Agents::PostAgent do
4
+    before do
5
+        @valid_params = {
6
+            :name => "somename",
7
+            :options => {
8
+                :post_url => "http://www.example.com",
9
+                :expected_receive_period_in_days => 1
10
+            } 
11
+        }
12
+
13
+    @checker = Agents::PostAgent.new(@valid_params)
14
+    @checker.user = users(:jane)
15
+    @checker.save!
16
+
17
+    @event = Event.new
18
+    @event.agent = agents(:jane_weather_agent)
19
+    @event.payload = {
20
+        :somekey => "somevalue",
21
+        :someotherkey => {
22
+            :somekey => "value"
23
+        }
24
+    }
25
+
26
+    @sent_messages = []
27
+    stub.any_instance_of(Agents::PostAgent).post_event { |uri,event| @sent_messages << event}
28
+    end
29
+
30
+    describe "#receive" do
31
+        it "checks if it can handle multiple events" do
32
+            event1 = Event.new
33
+            event1.agent = agents(:bob_weather_agent)
34
+            event1.payload = {
35
+                :xyz => "value1",
36
+                :message => "value2"
37
+            }
38
+
39
+            lambda {
40
+                @checker.receive([@event,event1])
41
+            }.should change { @sent_messages.length }.by(2)
42
+        end
43
+    end
44
+
45
+    describe "#working?" do
46
+        it "checks if events have been received within expected receive period" do
47
+            @checker.should_not be_working
48
+            Agents::PostAgent.async_receive @checker.id, [@event.id]
49
+            @checker.reload.should be_working
50
+            two_days_from_now = 2.days.from_now
51
+            stub(Time).now { two_days_from_now }  
52
+            @checker.reload.should_not be_working
53
+        end
54
+    end
55
+
56
+    describe "validation" do
57
+        before do
58
+            @checker.should be_valid
59
+        end
60
+
61
+        it "should validate presence of post_url" do
62
+            @checker.options[:post_url] = ""
63
+            @checker.should_not be_valid
64
+        end
65
+
66
+        it "should validate presence of expected_receive_period_in_days" do
67
+            @checker.options[:expected_receive_period_in_days] = ""
68
+            @checker.should_not be_valid
69
+        end
70
+    end
71
+end